# 爬取同花顺概念信息_ 头部已分类信息
import os
import sys
import zipfile

import requests
import pymysql
import traceback
import undetected_chromedriver as uc
from lxml import etree
from selenium.common import NoSuchElementException, TimeoutException
from selenium.webdriver.common.by import By
import time
from dbse.mysql.db_mysql_insert import addThsGNStocks
from dbse.mysql.db_mysql_insert import addStockThsSelException
from dbse.mysql.db_mysql_insert import addStockKpi
from dbse.mysql.db_mysql_insert import getDataBase
from dbse.mysql.db_mysql_insert import updateGnCompleted
from proxy.selenium_proxy_util import initChrome
from dbse.mysql.db_mysql_insert import deleteGnStocks
from dbse.mysql.db_entity_stock_ths_gn_stocks import StockThsGNStocks
from dbse.mysql.db_entity_stock_kpi_day import StockKpiDay
from dbse.mysql.db_entity_stock_ths_exception import StockThsSelException
from selenium.webdriver import ChromeOptions
from selenium.webdriver.support.ui import WebDriverWait
from sel.proxy.selenium_proxy_auth_plugin import create_proxy_auth_extension as cpae
from utils.StringUtil import simpleReplace
from utils.DateUtil import getTodayStr
from selenium.webdriver.support import expected_conditions as EC
from file.readFile import read
import datetime
import json
from sel.proxy.selenium_proxy_util import initSelWireChrome

def addException(db,stockCode):
    sql = 'INSERT INTO stock_ths_sel_exception (biz_code,sel_type,sta_date) VALUES (%s,%s,%s)'
    entity = StockThsSelException(stockCode,'1',getTodayStr())
    addStockThsSelException(db,sql,entity)

# 处理一页数据
def sel_gn_stocks(browser,db,add_sql,gn):
    gnCode = gn[0]                                              # 概念代码
    today = datetime.date.today().strftime("%Y-%m-%d")          # 时间
    content = browser.page_source
    # print(content)
    html = etree.fromstring(content,etree.HTMLParser())
    # result = html.xpath('//tbody//tr')                          # 股票数据
    result = html.xpath('//div[@id="maincont"]//tbody//tr')       # 股票数据 注意: 网页有多个版本 有的网页有两个tbody 所有要再通过div id=maincont 过滤一层
    for index, tr in enumerate(result):
        tds = tr.getchildren()
        # splitUrlItems = tds[1].find('a').get('href').split("/")
        # gnCode = splitUrlItems[len(splitUrlItems)-2]
        # print(f"====index:{index}")
        # print(tds[1].find('a').text)                # 股票代码
        # print(tds[2].find('a').text)                # 股票名称
        entity = StockThsGNStocks(tds[1].find('a').text,tds[2].find('a').text,today,gnCode)
        addThsGNStocks(db,add_sql,entity)

# 获取代理server 地址 有效期1-5分钟（私密代理）
def get_proxy():
    url = "https://dps.kdlapi.com/api/getdps/?secret_id=ondbxg8msovv7ury9qbe&signature=zft63syjut8u3og5m9j5wung50mwah5f&num=1&format=text&sep=1"
    proxy_address = requests.get(url)
    return proxy_address.text

def getPage(browser,stockCode,jys):
    url = f"https://quote.eastmoney.com/{jys}{stockCode}.html"
    browser.get(url)

def getEagerPage(browser,stockCode,jys):
    try:
        url = f"https://quote.eastmoney.com/{jys}{stockCode}.html"
        browser.get(url)
    except TimeoutException:
        print("browser get overtime")


def startWork(pJys,profile):
    db = getDataBase(profile)
    ### 查询所有概念  先判断是否还有未完成任务
    today = datetime.date.today().strftime("%Y-%m-%d")          # 时间
    query_sql = f'select * from (select t2.stock_code as tag, t1.* from stock_all t1 left  join stock_dfcf_kpi_day t2 on t1.stock_code = t2.stock_code and t2.sta_date = "{today}" where t1.jys = "{pJys}") list where tag is null '
    print(query_sql)
    add_sql = 'INSERT INTO stock_dfcf_kpi_day (stock_code, stock_name, price_begin, price_end, price_max, price_min, price_increase, price_increase_rate, per, pbr,  sta_date, turnover_ratio, amount, total_capital, trunover, price_yesterday, vol_ratio) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'
    cursor = db.cursor()
    cursor.execute(query_sql)
    stockAlls = cursor.fetchall()
    if len(stockAlls) == 0:
        print("=====================没有跟多需要处理的任务了=====================")
        return

    try:
        # 初始化浏览器
        # browser = initChrome(False,False,'https://quote.eastmoney.com')
        browser = initSelWireChrome(useProxy=True,isHeadless=False,host='https://quote.eastmoney.com',setTimeOut=True,ignorePic=True)
    except Exception as e:
        print("==========init chrome error============")
        startWork(pJys,profile)


    exceptionCount = 0
    for stock in stockAlls:
        print(f"====处理股票{stock[2]}")
        stockCode = stock[2]
        stockName = stock[4]
        jys = stock[5].lower()

        try:
            #抓取数据
            # getPage(browser,stockCode,jys)
            getEagerPage(browser,stockCode,jys)
            content = browser.page_source
            html = etree.fromstring(content,etree.HTMLParser())
            # 判断是否已退市
            error_state =  html.xpath('//div[@class="errorstate"]')
            if error_state:
                if error_state[0].text == '已退市':
                    print(f"==========股票{stockCode}:已退市")
                    continue
            # 判断是否科创板
            if stockCode.startswith("688"):
                # print(f"==========股票{stockCode}:是科创板")
                shoupan_result = html.xpath('//div[@class="zxj"]//span')
                zhangdie_result = html.xpath('//div[@class="zd"]/span')       # 两层span 嵌套 <span><span>xxx</span></span> 所以使用单 / 直接子元素
                # print(shoupan_result[0].find('span').text)                    #收盘
                # print(zhangdie_result[0].find('span').text)                   #涨跌
                # print(zhangdie_result[1].find('span').text)                   #涨跌百分比
                result = html.xpath('//div[@class="brief_info_c"]//tbody//tr')          #
                row1_tds = result[0].getchildren()
                row2_tds = result[1].getchildren()
                row3_tds = result[2].getchildren()
                # print(row1_tds[0].text)
                # print(row1_tds[5].find('span').find('span').text)           #今开
                # print(row1_tds[3].find('span').find('span').text)           #最高
                # print(row1_tds[7].find('span').find('span').text)           #换手
                # print(row1_tds[9].find('span').find('span').text)           #总市值
                # print(row2_tds[1].find('span').find('span').text)           #成交量
                # print(row2_tds[3].find('span').find('span').text)           #最低
                # print(row2_tds[5].find('span').find('span').text)           #昨收
                # print(row2_tds[7].find('span').find('span').text)           #量比
                # print(row2_tds[9].find('span').find('span').text)           #流通市值
                # print(row3_tds[1].find('span').find('span').text)           #成交额
                # print(row3_tds[3].find('span').find('span').text)           #震幅 （暂时不用）
                # print(row3_tds[7].find('span').find('span').text)           #市盈率
                # print(row3_tds[9].find('span').find('span').text)           #市净率
                stockKpiDay = StockKpiDay(stockCode,stockName,
                                          row1_tds[5].find('span').find('span').text,
                                          shoupan_result[0].find('span').text,
                                          row1_tds[3].find('span').find('span').text,
                                          row2_tds[3].find('span').find('span').text,
                                          zhangdie_result[0].find('span').text,
                                          simpleReplace(zhangdie_result[1].find('span').text),
                                          row3_tds[7].find('span').find('span').text,
                                          row3_tds[9].find('span').find('span').text,
                                          getTodayStr(),
                                          simpleReplace(row1_tds[7].find('span').find('span').text),
                                          simpleReplace(row3_tds[1].find('span').find('span').text),
                                          simpleReplace(row1_tds[9].find('span').find('span').text),
                                          simpleReplace(row2_tds[1].find('span').find('span').text),
                                          row2_tds[5].find('span').find('span').text,
                                          row2_tds[7].find('span').find('span').text
                                          )
                addStockKpi(db,add_sql,stockKpiDay)
            else:
                shoupan_result = html.xpath('//div[@class="zxj"]//span')
                zhangdie_result = html.xpath('//div[@class="zd"]/span')       # 两层span 嵌套 <span><span>xxx</span></span> 所以使用单 / 直接子元素
                # print(shoupan_result[0].find('span').text)                    #收盘
                # print(zhangdie_result[0].find('span').text)                   #涨跌
                # print(zhangdie_result[1].find('span').text)                   #涨跌百分比

                result = html.xpath('//div[@class="brief_info_c"]//tbody//tr')          #
                row1_tds = result[0].getchildren()
                row2_tds = result[1].getchildren()
                # print(row1_tds[0].text)
                # print(row1_tds[1].find('span').find('span').text)           #今开
                # print(row1_tds[3].find('span').find('span').text)           #最高
                # print(row1_tds[5].find('span').find('span').text)           #涨停
                # print(row1_tds[7].find('span').find('span').text)           #换手
                # print(row1_tds[9].find('span').find('span').text)           #成交量
                # print(row1_tds[11].find('span').find('span').text)           #市盈(动)
                # print(row1_tds[13].find('span').find('span').text)           #总市值
                #
                # print(row2_tds[1].find('span').find('span').text)           #昨收
                # print(row2_tds[3].find('span').find('span').text)           #最低
                # print(row2_tds[5].find('span').find('span').text)           #跌停
                # print(row2_tds[7].find('span').find('span').text)           #量比
                # print(row2_tds[9].find('span').find('span').text)           #成交额
                # print(row2_tds[11].find('span').find('span').text)           #市净
                # print(row2_tds[13].find('span').find('span').text)           #流通市值

                stockKpiDay = StockKpiDay(stockCode,stockName,
                                          row1_tds[1].find('span').find('span').text,
                                          shoupan_result[0].find('span').text,
                                          row1_tds[3].find('span').find('span').text,
                                          row2_tds[3].find('span').find('span').text,
                                          zhangdie_result[0].find('span').text,
                                          simpleReplace(zhangdie_result[1].find('span').text),
                                          row1_tds[11].find('span').find('span').text,
                                          row2_tds[11].find('span').find('span').text,
                                          getTodayStr(),
                                          simpleReplace(row1_tds[7].find('span').find('span').text),
                                          simpleReplace(row2_tds[9].find('span').find('span').text),
                                          simpleReplace(row1_tds[13].find('span').find('span').text),
                                          simpleReplace(row1_tds[9].find('span').find('span').text),
                                          row2_tds[1].find('span').find('span').text,
                                          row2_tds[7].find('span').find('span').text
                                          )
                addStockKpi(db,add_sql,stockKpiDay)
        except Exception as e:
            exceptionCount = exceptionCount+1
            # print(f"异常信息: {e}")
            # print("详细堆栈信息:")
            # traceback.print_exc()  # 打印完整的堆栈跟踪
            print(f"======================{stockCode}发生异常======================")
            addException(db,stockCode)
            time.sleep(1)
            ## 重启任务
            if(exceptionCount > 5):
                browser.quit()
                print("======================任务终止，等待重启======================")
                time.sleep(3)
                startWork(pJys,profile)

pJys = sys.argv[1:][0]
pProfile = sys.argv[1:][1]
startWork(pJys,pProfile)







